# -*- coding: utf-8 -*-
from scrapy import Spider, Request
from scrapy.selector import Selector
from datetime import datetime, timedelta
import json
import sys
import traceback
import urllib3
from datetime import datetime
from pymongo import MongoClient, UpdateOne
import pandas as pd


class TaobaoSpider(Spider):
    name = 'stock'
    db = MongoClient('mongodb://39.107.106.234:27017')['finaa_quant']
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
    op = 0

    def start_requests(self, date=None):
        DB_CONN = MongoClient('mongodb://39.107.106.234:27017')['final_quant']
        datetime_obj = datetime.now()
        if date is None:
            date = datetime_obj.strftime('%Y-%m-%d')
        codes = []
        # for type in ['ba', 'k']:
        #     while len(codes) == 0:
        #         code_cursor = DB_CONN.basic.find(
        #             {'date': date},
        #             projection={'code': True, '_id': False})
        #
        #         codes = [x['code'] for x in code_cursor]
        #
        #         datetime_obj = datetime_obj - timedelta(days=1)
        #         date = datetime_obj.strftime('%Y-%m-%d')
        #     url = 'http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?rtntype=1&token=4f1862fc3b5e77c150a2b985b12db0fd&id={0}&type=k&authorityType={1}&cb='
        #     for asa in codes:
        #         if asa[0:2] == '60':
        #             asa = asa + '1'
        #         elif asa[0:2] in ['30', '00']:
        #             asa = asa + '2'
        #         yield Request(url.format(asa, type), self.dual, meta={'code': asa[:-1], 'type': type})

        index_codes = ['000001', '000300']
        indexurl = 'http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?rtntype=1&token=4f1862fc3b5e77c150a2b985b12db0fd&cb=&id={0}1&type=k&iscr=false&_=1528087227767'
        for index in index_codes:
            yield Request(indexurl.format(index), self.dual_index, meta={'code': index})

    def dualamount(self, x):
        if x[-1] in ['亿', '万']:
            amount = float(x[0:-1])
            amount_unit = x[-1]
            if amount_unit == '亿':
                amount *= amount * 1e8
            elif amount_unit == '万':
                amount *= amount * 1e4
        else:
            amount = float(x)
        return amount

    def dual(self, response):
        body = str(response.body, encoding='utf-8')[1:]
        body = body.split('\n')
        if response.meta['type'] == 'ba':
            body = body[:-1]
        a = pd.DataFrame(body, )
        a = a[0].str.split(',', expand=True)
        if response.meta['type'] == 'ba':
            a.columns = ['date', 'open', 'close', 'high', 'low', 'volume', 'amount', 'index']
        else:
            a.columns = ['date', 'open', 'close', 'high', 'low', 'volume', 'amount', 'index', 'unknown']
        a['volume'] = a['volume'].apply(lambda x: int(x) * 100)
        a['amount'] = a['amount'].apply(self.dualamount).astype('int64')
        a['trade'] = a['volume'].apply(lambda x: True if x > 0 else False)
        a['code'] = response.meta['code']
        a = pd.DataFrame(a, columns=['date', 'open', 'close', 'high', 'low', 'volume', 'amount', 'trade', 'code'])
        a_250 = a.sort_values(by=['date'], ascending=False).head(250)
        # if response.meta['type']=='k':
        #     self.db.daily_k_all.insert(json.loads((a.T.to_json())).values())
        # elif response.meta['type']=='ba':
        #     self.db.daily_kba_all.insert(json.loads((a.T.to_json())).values())
        if response.meta['type'] == 'k':
            self.db.daily_k_250.insert(json.loads((a_250.T.to_json())).values())
        elif response.meta['type'] == 'ba':
            self.db.daily_kba_250.insert(json.loads((a_250.T.to_json())).values())
        print('code:{},done:{}'.format(response.meta['code'], len(a)))
        yield None

    def dual_index(self, response):
        body = str(response.body, encoding='utf-8')[1:]
        body = body.split('\n')
        a = pd.DataFrame(body, )
        a = a[0].str.split(',', expand=True)
        a.columns = ['date', 'open', 'close', 'high', 'low', 'volume', 'amount', 'index', 'unknown']
        a['volume'] = a['volume'].apply(lambda x: int(x) * 100)
        a['amount'] = a['amount'].apply(self.dualamount).astype('int64')
        a['trade'] = a['volume'].apply(lambda x: True if x > 0 else False)
        a['code'] = response.meta['code']
        a = pd.DataFrame(a, columns=['date', 'open', 'close', 'high', 'low', 'amount', 'code'])
        a_250 = a.sort_values(by=['date'], ascending=False).head(250)
        self.db.daily_index_all.insert(json.loads((a.T.to_json())).values())
        self.db.daily_index_250.insert(json.loads((a_250.T.to_json())).values())
        print('code:{},done:{}'.format(response.meta['code'], len(a)))
        yield None
